#include<vector>
#include<ctime>
#include<cmath>
#include<string>
#include<fstream>
#include"Util.h"
#include"StringPool.h"
#include"variable.h"
#include"DependencyInstance.h"
#include"KBestParseForest.h"
#include"DependencyPipe.h"
using namespace std;

DependencyPipe::DependencyPipe(ParserOptions* options){
	labeled = options->labeled;
	isCONLL = true;
	this->options = options;
	if(!((*(options->format)) == "CONLL")){
		isCONLL = false;
	}

	typesInt = NULL;

	this->map_size = options->map_size;
	this->dataAlphabet = Alphabet(map_size);
	this->typeAlphabet = Alphabet(30);

	depReader = DependencyReader::createDependencyReader(*(options->format), options->discourseMode, labeled, options->hasEnd);
	depWriter = NULL;
}

DependencyPipe::~DependencyPipe(){
	delete(depReader);
	if(depWriter != NULL){
		delete(depWriter);
	}
	if(typesInt != NULL){
		delete(typesInt);
	}
}

void DependencyPipe::initInputFile(std::string *file){
	depReader->startReading(file->c_str());
}

void DependencyPipe::initOutputFile(std::string *file){
	depWriter = DependencyWriter::createDependencyWriter(*(options->format), labeled, options->hasEnd);
	depWriter->startWriting(file->c_str());
}

void DependencyPipe::outputInstance(vector<string*> &forms, vector<string*> &lemma, vector<string*> &cpos, vector<string*> &pos, vector<string*> &labs, vector<int> &heads){
	depWriter->write(forms, lemma, cpos, pos, labs, heads);
}

void DependencyPipe::close(){
	if(depWriter != NULL){
		depWriter->finishWriting();
	}
	depReader->finishReading();
}

string* DependencyPipe::getType(int typeIndex){
	return types[typeIndex];
}

DependencyInstance* DependencyPipe::nextInstance(){
	DependencyInstance* instance = depReader->getNext();
	if(instance == NULL || instance->forms == NULL){
		return NULL;
	}
	return instance;
}

vector<int>* DependencyPipe::createInstances(string *file, std::string *featFileName){
	createAlphabet(file);

	cout<<"Num Features: "<<dataAlphabet.size()<<endl;

	depReader->startReading(file->c_str());

	vector<int>* lengths = new vector<int>();
	ObjectWriter* out = NULL;
	if(options->createForest){
		out = new ObjectWriter(featFileName->c_str());
	}

	DependencyInstance* instance = depReader->getNext();
	cout<<"Creating Feature Vector Instances: "<<endl;

	int num1 = 0;
	while(instance != NULL){
		printf("%d ", num1++);
		fflush(stdout);
		instance->setFeatureVector(createFeatureVector(instance, true));
		vector<string*>* labs = instance->deprels;
		vector<int>* heads = instance->heads;
		string* spans = pool->get();
		
		int length = (int)(heads->size());
		for(int i = 1; i < length; i++){
			string* str = Util::intToString((*heads)[i]);
			spans->append(*str);
			pool->push_back(str);

			spans->append("|");

			str = Util::intToString(i);
			spans->append(*str);
			pool->push_back(str);;

			spans->append(":");

			bool flag;
			string* tmp = pool->get(*((*labs)[i]));
			int j = typeAlphabet.lookupIndex(tmp, flag);
			if(flag){
				pool->push_back(tmp);
			}
			str = Util::intToString(j);
			spans->append(*str);
			pool->push_back(str);
			spans->append(" ");
		}
		string tmp = spans->substr(0, spans->length() -1);
		instance->actParserTree = pool->get(tmp);
		
		lengths->push_back(instance->length());
		if(options->createForest){
			writeInstance(instance, *out);
		}
		pool->push_back(spans);
		delete(instance);
		instance = depReader->getNext();
	}
	
	if(options->createForest){
		delete(out);
	}
	depReader->finishReading();
	cout<<endl;
	return lengths;
}

vector<int>* DependencyPipe::createInstances(std::string *file, std::vector<DependencyInstance*> *instList){
	createAlphabet(file);

	depReader->startReading(file->c_str());

	vector<int>* lengths = new vector<int>();

	DependencyInstance* instance = depReader->getNext();

	while(instance != NULL){
		vector<string*>* labs = instance->deprels;
		vector<int>* heads = instance->heads;
		string* spans = pool->get();
		
		int length = (int)(heads->size());
		for(int i = 1; i < length; i++){
			string* str = Util::intToString((*heads)[i]);
			spans->append(*str);
			pool->push_back(str);

			spans->append("|");

			str = Util::intToString(i);
			spans->append(*str);
			pool->push_back(str);;

			spans->append(":");

			bool flag;
			string* tmp = pool->get(*((*labs)[i]));
			int j = typeAlphabet.lookupIndex(tmp, flag);
			if(flag){
				pool->push_back(tmp);
			}
			str = Util::intToString(j);
			spans->append(*str);
			pool->push_back(str);
			spans->append(" ");
		}
		string tmp = spans->substr(0, spans->length() -1);
		instance->actParserTree = pool->get(tmp);
		
		lengths->push_back(instance->length());
		pool->push_back(spans);
		if(options->createForest){
			instList->push_back(instance);
		}
		else{
			delete(instance);
		}
		instance = depReader->getNext();
	}
	depReader->finishReading();
	return lengths;
}

void DependencyPipe::createAlphabet(std::string *file){
	cout<<"Creating Alphabet ... ";
	depReader->startReading(file->c_str());

	DependencyInstance* instance = depReader->getNext();
	while(instance != NULL){
		vector<string*>* labs = instance->deprels;
		int length = (int)(labs->size());
		for(int i = 0; i < length; i++){
			bool flag;
			string* tmp = pool->get(*((*labs)[i]));
			typeAlphabet.lookupIndex(tmp, flag);
			if(flag){
				pool->push_back(tmp);
			}
		}
		instance->setFeatureVector(createFeatureVector(instance, false));
		delete(instance);
		instance = NULL;
		instance = depReader->getNext();
	}
	depReader->finishReading();
	closeAlphabets();
	cout<<"Done."<<endl;
}

void DependencyPipe::closeAlphabets(){
	dataAlphabet.stopGrowth();
	typeAlphabet.stopGrowth();

	types = vector<string*>(typeAlphabet.size());
	vector<string*>* keys = typeAlphabet.toArray();

	int length = (int)(keys->size());
	for(int i = 0; i < length; i++){
		bool flag;
		int indx = typeAlphabet.lookupIndex((*keys)[i], flag);
		types[indx] = (*keys)[i];
	}
	if(labeled){
		string tmp = string("ROOT");
		string* str = pool->get(tmp);
		bool flag;
		KBestParseForest::setRootType(typeAlphabet.lookupIndex(str, flag));
		if(flag){
			pool->push_back(str);
		}
	}
	else{
		KBestParseForest::setRootType(0);
	}
	delete(keys);
}

// add with default 1.0
bool DependencyPipe::add(std::string *feat, FeatureVector *fv){
	bool flag = true;
	int num = dataAlphabet.lookupIndex(feat, flag);
	
	if(num >= 0 && fv != NULL){
		fv->add(num, 1.0);
	}
	return flag;
}

FeatureVector* DependencyPipe::createFeatureVector(DependencyInstance *instance, bool genFeature){
	const int instanceLength = instance->length();

	vector<string*>* labs = instance->deprels;
	vector<int>* heads = instance->heads;

	FeatureVector* fv = genFeature ? new FeatureVector() : NULL;

	for(int i = 0; i < instanceLength; i++){
		if((*heads)[i] == -1){
			continue;
		}
		int small = i < (*heads)[i] ? i : (*heads)[i];
		int large = i > (*heads)[i] ? i : (*heads)[i];
		bool attR = i < (*heads)[i] ? false : true;

		addCoreFeatures(instance, small, large, attR, fv);
		
		if(labeled){
			addLabeledFeatures(instance, i, (*labs)[i], attR, true, fv);
			addLabeledFeatures(instance, (*heads)[i], (*labs)[i], attR, false, fv);
		}
		
	}
	
	addExtendedFeatures(instance, fv);
	return fv;
}

void DependencyPipe::addExtendedFeatures(DependencyInstance *instance, FeatureVector *fv){};

void DependencyPipe::addCoreFeatures(DependencyInstance *instance, int small, int large, bool attR, FeatureVector *fv){
	vector<string*>* forms = instance->forms;
	vector<string*>* pos = instance->postags;
	vector<string*>* posA = instance->cpostags;
	vector<string*>* lemmas = instance->lemmas;

	string att = attR ? "RA" : "LA";

	
	int dist = abs(large - small);
	string* distBool = NULL;
	string tmp;
	if (dist > 10){
		tmp = string("10");
		distBool = pool->get(tmp);
	}
	else if (dist > 5){
		tmp = string("5");
		distBool = pool->get(tmp);
	}
	else{
		distBool = Util::intToString(dist - 1);
	}
	

	string attDist = "&" + att + "&" + *distBool;
	string type = "POS";
	addLinearFeatures(type, pos, small, large, attDist, fv);
	type = "CPOS";
	addLinearFeatures(type, posA, small, large, attDist, fv);

	type = "FORM";
	addLinearFeatures(type, forms, small, large, attDist, fv);
	type = "LEMMA";
	addLinearFeatures(type, lemmas, small, large, attDist, fv);

	pool->push_back(distBool);

	// /////////////////////////////////////////////////////////////
	
	int headIndex = small;
	int childIndex = large;
	if(!attR){
		headIndex = large;
		childIndex = small;
	}
	
	type = "HC";
	addTwoObsFeatures(type, (*forms)[headIndex], (*pos)[headIndex], (*forms)[childIndex], (*pos)[childIndex], attDist, fv);

	if(isCONLL){
		type = "HCA";
		addTwoObsFeatures(type, (*forms)[headIndex], (*posA)[headIndex], (*forms)[childIndex], (*posA)[childIndex], attDist, fv);
		
		type = "HCC";
		addTwoObsFeatures(type, (*(instance->lemmas))[headIndex], (*pos)[headIndex], (*(instance->lemmas))[childIndex], (*pos)[childIndex], attDist, fv);
		
		type = "HCD";
		addTwoObsFeatures(type, (*(instance->lemmas))[headIndex], (*posA)[headIndex], (*(instance->lemmas))[childIndex], (*posA)[childIndex], attDist, fv);
		

		if(options->discourseMode){
			// Note: The features invoked here are designed for
			// discourse parsing (as opposed to sentential
			// parsing). It is conceivable that they could help for
			// sentential parsing, but current testing indicates that
			// they hurt sentential parsing performance.

			addDiscourseFeatures(instance, small, large, headIndex, childIndex, attDist, fv);
		}
		else{
			// Add in features from the feature lists. It assumes
			// the feature lists can have different lengths for
			// each item. For example, nouns might have a
			// different number of morphological features than
			// verbs.
			int length1 = (int)((*(instance->feats))[headIndex]->size());
			int length2 = (int)((*(instance->feats))[childIndex]->size());
			for(int i= 0; i < length1; i++){
				string* ii = Util::intToString(i);
				for(int j= 0; j < length2; j++){
					string* jj = Util::intToString(j);
					tmp = "FF" + *ii + "*" + *jj;
					addTwoObsFeatures(tmp, (*(instance->forms))[headIndex], (*(*(instance->feats))[headIndex])[i], (*(instance->forms))[childIndex], (*(*(instance->feats))[childIndex])[j], attDist, fv);
					tmp = "LF" + *ii + "*" + *jj;
					addTwoObsFeatures(tmp, (*(instance->lemmas))[headIndex], (*(*(instance->feats))[headIndex])[i], (*(instance->lemmas))[childIndex], (*(*(instance->feats))[childIndex])[j], attDist, fv);
					pool->push_back(jj);
				}
				pool->push_back(ii);
			}
		}
	}
	else{
		// We are using the old MST format. Pick up stem features
		// the way they used to be done. This is kept for
		// replicability of results for old versions.
		int hL = (int)((*forms)[headIndex]->size());
		int cL = (int)((*forms)[childIndex]->size());

		if(hL > 5 || cL > 5){
			addOldMSTStemFeatures((*(instance->lemmas))[headIndex], (*pos)[headIndex], (*(instance->lemmas))[childIndex], (*pos)[childIndex], attDist, hL, cL, fv);
		}
	}
	
}

void DependencyPipe::addLinearFeatures(std::string &type, std::vector<string*> *obsVals, int first, int second, std::string &attachDistance, FeatureVector *fv){

	string pLeft = first > 0 ? *((*obsVals)[first - 1]) : "STR";
	string pRight = second < (int)(obsVals->size()) - 1 ? *((*obsVals)[second + 1]) : "END";

	string pLeftRight = first < second - 1 ? *((*obsVals)[first + 1]) : "MID";
	string pRightLeft = second > first + 1 ? *((*obsVals)[second - 1]) : "MID";

	// feature posR posMid posL
	string featPos = type + "PC=" + *((*obsVals)[first]) + " " + *((*obsVals)[second]);

	for(int i = first + 1; i < second; i++){
		string* str = (*obsVals)[i];
		string tmp = featPos + " " + *str;
		string* allPos = pool->get(tmp);

		bool flag;
		flag = add(allPos, fv);
		string* allPos2;
		if(flag){
			allPos->append(attachDistance);
			allPos2 = allPos;
		}
		else{
			tmp = *allPos + attachDistance;
			allPos2 = pool->get(tmp);
		}
		flag = add(allPos2, fv);
		if(flag){
			pool->push_back(allPos2);
		}
	}

	if(type == "POS" || type == "CPOS"){
		string tmpp = type + "PT";
		addCorePosFeatures(tmpp, pLeft, (*obsVals)[first], pLeftRight, pRightLeft, (*obsVals)[second], pRight, attachDistance, fv);
	}
}

void DependencyPipe::addCorePosFeatures(std::string &prefix, std::string &leftOf1, std::string *one, std::string &rightOf1, std::string &leftOf2, std::string *two, std::string &rightOf2, std::string &attachDistance, FeatureVector *fv){
	// feature posL-1 posL posR posR+1

	string tmp = prefix + "=" + leftOf1 + " " + *one + " " + *two + "*" + attachDistance;
	string* feat = pool->get(tmp);
	bool flag = true;
	flag = add(feat, fv);
	
	//
	if(flag){
		*feat = prefix + "1=" + leftOf1 + " " + *one + " " + *two;
	}
	else{
		tmp = prefix + "1=" + leftOf1 + " " + *one + " " + *two;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append(" " + rightOf2);
	}
	else{
		tmp = *feat + " " + rightOf2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2=" + leftOf1 + " " + *two + " " + rightOf2;
	}
	else{
		tmp = prefix + "2=" + leftOf1 + " " + *two + " " + rightOf2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "3=" + leftOf1 + " " + *one + " " + rightOf2;
	}
	else{
		tmp = prefix + "3=" + leftOf1 + " " + *one + " " + rightOf2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	
	//
	if(flag){
		*feat = prefix + "4=" + *one + " " + *two + " " + rightOf2;
	}
	else{
		tmp = prefix + "4=" + *one + " " + *two + " " + rightOf2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		pool->push_back(feat);
	}

	// ///////////////////////////////////////////////////////////
	prefix = "A" + prefix;

	// feature posL posL+1 posR-1 posR
	tmp = prefix + "1=" + *one + " " + rightOf1 + " " + leftOf2 + "*" + attachDistance;
	feat = pool->get(tmp);
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "1=" + *one + " " + rightOf1 + " " + leftOf2;
	}
	else{
		tmp = prefix + "1=" + *one + " " + rightOf1 + " " + leftOf2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append(" "+ *two);
	}
	else{
		tmp = *feat + " " + *two;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2=" + *one + " " + rightOf1 + " " + *two;
	}
	else{
		tmp = prefix + "2=" + *one + " " + rightOf1 + " " + *two;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "3=" + *one + " " + leftOf2 + " " + *two;
	}
	else{
		tmp = prefix + "3=" + *one + " " + leftOf2 + " " + *two;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "4=" + rightOf1 + " " + leftOf2 + " " + *two;
	}
	else{
		tmp = prefix + "4=" + rightOf1 + " " + leftOf2 + " " + *two;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		pool->push_back(feat);
	}

	// //////////////////////////////////////////////////////////
	prefix = "B" + prefix;

	// // feature posL-1 posL posR-1 posR
	tmp = prefix + "1=" + leftOf1 + " " + *one + " " + leftOf2 + " " + *two;
	feat = pool->get(tmp);
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		pool->push_back(feat);
	}

	// // feature posL posL+1 posR posR+1
	tmp = prefix + "2=" + *one + " " + rightOf1 + " " + *two + " " + rightOf2;
	feat = pool->get(tmp);
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		pool->push_back(feat);
	}
}

/**
	 * Add features for two items, each with two observations, e.g. head, head
	 * pos, child, and child pos.
	 * 
	 * The use of StringBuilders is not yet as efficient as it could be, but
	 * this is a start. (And it abstracts the logic so we can add other features
	 * more easily based on other items and observations.)
 **/
void DependencyPipe::addTwoObsFeatures(std::string &prefix, std::string *item1F1, std::string *item1F2, std::string *item2F1, std::string *item2F2, std::string &attachDistance, FeatureVector *fv){
	
	string tmp = prefix + "2FF1=" + *item1F1;
	string* feat = pool->get(tmp);
	bool flag = true;
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF1=" + *item1F1 + " " + *item1F2;
	}
	else{
		tmp = prefix + "2FF1=" + *item1F1 + " " + *item1F2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF1=" + *item1F1 + " " + *item1F2 + " " + *item2F2;
	}
	else{
		tmp = prefix + "2FF1=" + *item1F1 + " " + *item1F2 + " " + *item2F2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF1=" + *item1F1 + " " + *item1F2 + " " + *item2F2 + " " + *item2F1;
	}
	else{
		tmp = prefix + "2FF1=" + *item1F1 + " " + *item1F2 + " " + *item2F2 + " " + *item2F1;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF2=" + *item1F1 + " " + *item2F1;
	}
	else{
		tmp = prefix + "2FF2=" + *item1F1 + " " + *item2F1;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF3=" + *item1F1 + " " + *item2F2;
	}
	else{
		tmp = prefix + "2FF3=" + *item1F1 + " " + *item2F2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF4=" + *item1F2 + " " + *item2F1;
	}
	else{
		tmp = prefix + "2FF4=" + *item1F2 + " " + *item2F1;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF4=" + *item1F2 + " " + *item2F1 + *item2F2;
	}
	else{
		tmp = prefix + "2FF4=" + *item1F2 + " " + *item2F1 + *item2F2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF5=" + *item1F2 + " " + *item2F2;
	}
	else{
		tmp = prefix + "2FF5=" + *item1F2 + " " + *item2F2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF6=" + *item2F1 + " " + *item2F2;
	}
	else{
		tmp = prefix + "2FF6=" + *item2F1 + " " + *item2F2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF7=" + *item1F2;
	}
	else{
		tmp = prefix + "2FF7=" + *item1F2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF8=" + *item2F1;
	}
	else{
		tmp = prefix + "2FF8=" + *item2F1;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);

	//
	if(flag){
		*feat = prefix + "2FF9=" + *item2F2;
	}
	else{
		tmp = prefix + "2FF9=" + *item2F2;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		feat->append("*" + attachDistance);
	}
	else{
		tmp = *feat + "*" + attachDistance;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		pool->push_back(feat);
	}
}

void DependencyPipe::addLabeledFeatures(DependencyInstance *instance, int word, std::string *type, bool attR, bool childFeatures, FeatureVector *fv){
	if(!labeled){
		return;
	}

	vector<string*>* forms = instance->forms;
	vector<string*>* pos = instance->postags;

	string att;
	if(attR){
		att = "RA";
	}
	else{
		att = "LA";
	}
	if(childFeatures){
		att = att + "&" + "true";
	}
	else{
		att = att + "&" + "false";
	}

	string* str;
	str = (*forms)[word];
	string w = *str;
	str = (*pos)[word];
	string wP = *str;

	string wPm1 = word > 0 ? str = (*pos)[word - 1], *str : "STR";
	string wPp1 = word < (int)(pos->size()) - 1 ? str = (*pos)[word + 1], *str : "END";

	string tmp = "NTS1=" + *type + "&" + att;
	bool flag = true;
	string* feat = pool->get(tmp);
	flag = add(feat, fv);
	if(flag){
		*feat = "ANTS1=" + *type;
	}
	else{
		tmp = "ANTS1=" + *type;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		pool->push_back(feat);
	}

	for(int i = 0; i < 2; i++){
		string suff = i < 1 ? "&" + att : "";
		suff = "&" + *type + suff;

		tmp = "NTH=" + w + " " + wP + suff;
		feat = pool->get(tmp);
		flag = add(feat, fv);
		if(flag){
			*feat = "NTI=" + wP + suff;
		}
		else{
			tmp = "NTI=" + wP + suff;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "NTIA=" + wPm1 + " " + wP + suff;
		}
		else{
			tmp = "NTIA=" + wPm1 + " " + wP + suff;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "NTIB=" + wP + " " + wPp1 + suff;
		}
		else{
			tmp = "NTIB=" + wP + " " + wPp1 + suff;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "NTIC=" + wPm1 + " " + wP + " " + wPp1 + suff;
		}
		else{
			tmp = "NTIC=" + wPm1 + " " + wP + " " + wPp1 + suff;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "NTJ=" + w + suff;
		}
		else{
			tmp = "NTJ=" + w + suff;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			pool->push_back(feat);
		}
	}
}

void DependencyPipe::addDiscourseFeatures(DependencyInstance *instance, int small, int large, int headIndex, int childIndex, std::string &attDist, FeatureVector *fv){

	string type = "FORM";
	addLinearFeatures(type, instance->forms, small, large, attDist, fv);

	type = "LEMMA";
	addLinearFeatures(type, instance->lemmas, small, large, attDist, fv);

	type = "HCB1";
	addTwoObsFeatures(type, (*(instance->forms))[headIndex], (*(instance->lemmas))[headIndex], (*(instance->forms))[childIndex], (*(instance->lemmas))[childIndex], attDist, fv);

	type = "HCB2";
	addTwoObsFeatures(type, (*(instance->forms))[headIndex], (*(instance->lemmas))[headIndex], (*(instance->forms))[childIndex], (*(instance->postags))[childIndex], attDist, fv);

	type = "HCB3";
	addTwoObsFeatures(type, (*(instance->forms))[headIndex], (*(instance->lemmas))[headIndex], (*(instance->forms))[childIndex], (*(instance->cpostags))[childIndex], attDist, fv);

	type = "HC2";
	addTwoObsFeatures(type, (*(instance->forms))[headIndex], (*(instance->postags))[headIndex], (*(instance->forms))[childIndex], (*(instance->cpostags))[childIndex], attDist, fv);

	type = "HCC2";
	addTwoObsFeatures(type, (*(instance->lemmas))[headIndex], (*(instance->postags))[headIndex], (*(instance->lemmas))[childIndex], (*(instance->cpostags))[childIndex], attDist, fv);

	// // Use this if your extra feature lists all have the same length.
	for (int i = 0; i < (int)((instance->feats)->size()); i++) {
		string* ii = Util::intToString(i);

		string tmp = "F" + *ii;
		addLinearFeatures(tmp, (*(instance->feats))[i], small, large, attDist, fv);

		tmp = "FF" + *ii;
		addTwoObsFeatures(tmp, (*(instance->forms))[headIndex], (*((*(instance->feats))[i]))[headIndex], (*(instance->forms))[childIndex], (*((*(instance->feats))[i]))[childIndex], attDist, fv);

		tmp = "LF" + *ii;
		addTwoObsFeatures(tmp, (*(instance->lemmas))[headIndex], (*((*(instance->feats))[i]))[headIndex], (*(instance->lemmas))[childIndex], (*((*(instance->feats))[i]))[childIndex], attDist, fv);

		tmp = "PF" + *ii;
		addTwoObsFeatures(tmp, (*(instance->postags))[headIndex], (*((*(instance->feats))[i]))[headIndex], (*(instance->postags))[childIndex], (*((*(instance->feats))[i]))[childIndex], attDist, fv);

		tmp = "CPF" + *ii;
		addTwoObsFeatures(tmp, (*(instance->cpostags))[headIndex], (*((*(instance->feats))[i]))[headIndex], (*(instance->cpostags))[childIndex], (*((*(instance->feats))[i]))[childIndex], attDist, fv);

		int length = (int)((instance->feats)->size());
		for (int j = i + 1; j < length; j++) {
			string* jj = Util::intToString(j);

			tmp = "CPF" + *ii + "_" + *jj;
			addTwoObsFeatures(tmp, (*((*(instance->feats))[i]))[headIndex], (*((*(instance->feats))[j]))[headIndex], (*((*(instance->feats))[i]))[childIndex], (*((*(instance->feats))[j]))[childIndex], attDist, fv);

			pool->push_back(jj);
		}

		length = (int)((instance->feats)->size());
		for (int j = 0; j < length; j++) {
			string* jj = Util::intToString(j);

			tmp = "XFF" + *ii + "_" + *jj;
			addTwoObsFeatures(tmp, (*(instance->forms))[headIndex], (*((*(instance->feats))[i]))[headIndex], (*(instance->forms))[childIndex], (*((*(instance->feats))[j]))[childIndex], attDist, fv);

			tmp = "XLF" + *ii + "_" + *jj;
			addTwoObsFeatures(tmp, (*(instance->lemmas))[headIndex], (*((*(instance->feats))[i]))[headIndex], (*(instance->lemmas))[childIndex], (*((*(instance->feats))[j]))[childIndex], attDist, fv);

			tmp = "XPF" + *ii + "_" + *jj;
			addTwoObsFeatures(tmp, (*(instance->postags))[headIndex], (*((*(instance->feats))[i]))[headIndex], (*(instance->postags))[childIndex], (*((*(instance->feats))[j]))[childIndex], attDist, fv);

			tmp = "XCF" + *ii + "_" + *jj;
			addTwoObsFeatures(tmp, (*(instance->cpostags))[headIndex], (*((*(instance->feats))[i]))[headIndex], (*(instance->cpostags))[childIndex], (*((*(instance->feats))[j]))[childIndex], attDist, fv);

			pool->push_back(jj);
		}
		pool->push_back(ii);
	}

	// Test out relational features
	if (options->useRelationalFeatures) {

		// for (int rf_index=0; rf_index<2; rf_index++) {
		int length = (int)((instance->relFeats)->size());
		for (int rf_index = 0; rf_index < length; rf_index++) {
			
			string* rf_index_s = Util::intToString(rf_index);
			string tmp = "H2C" + *rf_index_s + ((*(instance->relFeats))[rf_index])->getFeature(headIndex, childIndex);
			string* headToChild = pool->get(tmp);
			pool->push_back(rf_index_s);

			type = "RFA1";
			addTwoObsFeatures(type, (*(instance->forms))[headIndex], (*(instance->lemmas))[headIndex], (*(instance->postags))[childIndex], headToChild, attDist, fv);

			type = "RFA2";
			addTwoObsFeatures(type, (*(instance->postags))[headIndex], (*(instance->cpostags))[headIndex], (*(instance->forms))[childIndex], headToChild, attDist, fv);

			type = "RFA3";
			addTwoObsFeatures(type, (*(instance->lemmas))[headIndex], (*(instance->postags))[headIndex], (*(instance->forms))[childIndex], headToChild, attDist, fv);

			type = "RFB1";
			addTwoObsFeatures(type, headToChild, (*(instance->postags))[headIndex], (*(instance->forms))[childIndex], (*(instance->lemmas))[childIndex], attDist, fv);

			type = "RFB2";
			addTwoObsFeatures(type, headToChild, (*(instance->forms))[headIndex], (*(instance->postags))[childIndex], (*(instance->cpostags))[childIndex], attDist, fv);

			type = "RFB3";
			addTwoObsFeatures(type, headToChild, (*(instance->forms))[headIndex], (*(instance->lemmas))[childIndex], (*(instance->postags))[childIndex], attDist, fv);

			pool->push_back(headToChild);
		}
	}
}

void DependencyPipe::fillFeatureVectors(DependencyInstance *instance, std::vector<vector<vector<double> > > &probs, std::vector<vector<vector<vector<double> > > > &nt_probs, Parameters *params){
	const int instanceLength = instance->length();

	for(int w1 = 0; w1 < instanceLength; w1++){
		for (int w2 = w1 + 1; w2 < instanceLength; w2++) {
			for (int ph = 0; ph < 2; ph++) {
				bool attR = ph == 0 ? true : false;
				FeatureVector fv = FeatureVector();
				addCoreFeatures(instance, w1, w2, attR, &fv);
				double prodProb = params->getScore(&fv);
				probs[w1][w2][ph] = prodProb;
			}
		}
	}

	if(labeled){
		for (int w1 = 0; w1 < instanceLength; w1++) {
			int len = (int)(types.size());
			for (int t = 0; t < len; t++) {
				string* type = types[t];
				for (int ph = 0; ph < 2; ph++) {
					bool attR = ph == 0 ? true : false;
					for (int ch = 0; ch < 2; ch++) {
						bool child = ch == 0 ? true : false;
						FeatureVector fv = FeatureVector();
						addLabeledFeatures(instance, w1, type, attR, child, &fv);
						double nt_prob = params->getScore(&fv);
						nt_probs[w1][t][ph][ch] = nt_prob;
					}
				}
			}
		}
	}
}

/**
	* Write an instance to an output stream for later reading.
	* 
**/
void DependencyPipe::writeInstance(DependencyInstance *instance, ObjectWriter &out){
	int instanceLength = instance->length();

	for (int w1 = 0; w1 < instanceLength; w1++) {
		for (int w2 = w1 + 1; w2 < instanceLength; w2++) {
			for (int ph = 0; ph < 2; ph++) {
				bool attR = ph == 0 ? true : false;
				FeatureVector prodFV = FeatureVector();
				addCoreFeatures(instance, w1, w2, attR, &prodFV);
				int* keys = prodFV.keys();
				out.writeArray(keys, (int)(prodFV.size()));
				delete[](keys);
			}
		}
	}
	out.writeInt(-3);

	if (labeled) {
		for (int w1 = 0; w1 < instanceLength; w1++) {
			int len = (int)(types.size());
			for (int t = 0; t < len; t++) {
				string* type = types[t];
				for (int ph = 0; ph < 2; ph++) {
					bool attR = ph == 0 ? true : false;
					for (int ch = 0; ch < 2; ch++) {
						bool child = ch == 0 ? true : false;
						FeatureVector prodFV = FeatureVector();
						addLabeledFeatures(instance, w1, type, attR, child, &prodFV);
						int* keys = prodFV.keys();
						out.writeArray(keys, (int)(prodFV.size()));
						delete[](keys);
					}
				}
			}
		}
		out.writeInt(-3);
	}

	writeExtendedFeatures(instance, out);

	int* v = instance->fv->keys();
	out.writeArray(v, (int)(instance->fv->size()));
	delete[](v);
	out.writeInt(-4);

	instance->writeObject(out);
	out.writeInt(-1);
}

/**
	* Override this method if you have extra features that need to be written
	* to disk. For the basic DependencyPipe, nothing happens.
	* 
 */
void DependencyPipe::writeExtendedFeatures(DependencyInstance *instance, ObjectWriter &out){}

double DependencyPipe:: getScore(double *parameters, int *keys, int size){
	double score = 0.0;
	for(int i = 0; i < size; i++){
		score += parameters[keys[i]];
	}
	return score;
}


/**
	* Read an instance from an input stream.
	* 
**/
DependencyInstance* DependencyPipe::readInstance(ObjectReader &in, int length, std::vector<vector<vector<double> > > &probs, std::vector<vector<vector<vector<double> > > > &nt_probs, Parameters *params){
	// Get production crap.
	for (int w1 = 0; w1 < length; w1++) {
		for (int w2 = w1 + 1; w2 < length; w2++) {
			for (int ph = 0; ph < 2; ph++) {
				int *keys = NULL;
				int size = in.readArray(keys);
				probs[w1][w2][ph] = getScore(params->parameters, keys, size);
				delete[](keys);
			}
		}
	}
	int last = in.readInt();
	if(last != -3){
		cout<<"Error reading file."<<endl;
		exit(0);
	}

	if (labeled) {
		for (int w1 = 0; w1 < length; w1++) {
			int len = (int)(types.size());
			for (int t = 0; t < len; t++) {
				for (int ph = 0; ph < 2; ph++) {
					for (int ch = 0; ch < 2; ch++) {
						int *keys = NULL;
						int size = in.readArray(keys);
						nt_probs[w1][t][ph][ch] = getScore(params->parameters, keys, size);
						delete[](keys);
					}
				}
			}
		}
		last = in.readInt();
		if (last != -3) {
			cout<<"Error reading file."<<endl;
			exit(0);
		}
	}

	int *keys = NULL;
	int size = in.readArray(keys);
	FeatureVector* nfv = new FeatureVector();
	nfv->construct(keys, size);
	delete[](keys);
	last = in.readInt();
	if (last != -4) {
		cout<<"Error reading file."<<endl;
		exit(0);
	}

	DependencyInstance* marshalledDI = new DependencyInstance();
	marshalledDI->readObject(in);
	marshalledDI->setFeatureVector(nfv);

	last = in.readInt();
	if (last != -1) {
		cout<<"Error reading file."<<endl;
		exit(0);
	}
	return marshalledDI;
}

void DependencyPipe::addOldMSTStemFeatures(std::string *hLemma, std::string *headP, std::string *cLemma, std::string *childP, std::string &attDist, int hL, int cL, FeatureVector *fv){
	string all = *hLemma + " " + *headP + " " + *cLemma + " " + *childP;
	string hPos = *headP + " " + *cLemma + " " + *childP;
	string cPos = *hLemma + " " + *headP + " " + *childP;
	string hP = *headP + " " + *cLemma;
	string cP = *hLemma + " " + *childP;
	string oPos = *headP + " " + *childP;
	string oLex = *hLemma + " " + *cLemma;

	string tmp = "SA=" + all + attDist;
	string* feat = pool->get(tmp);
	bool flag;
	flag = add(feat, fv);
	if(flag){
		*feat = "SF=" + oLex + attDist;
	}
	else{
		tmp = "SF=" + oLex + attDist;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		*feat = "SAA=" + all;
	}
	else{
		tmp = "SAA=" + all;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		*feat = "SFF=" + oLex;
	}
	else{
		tmp = "SFF=" + oLex;
		feat = pool->get(tmp);
	}
	flag = add(feat, fv);
	if(flag){
		pool->push_back(feat);
	}

	if (cL > 5) {
		tmp = "SB=" + hPos + attDist;
		feat = pool->get(tmp);
		flag = add(feat, fv);
		if(flag){
			*feat = "SD=" + hP + attDist;
		}
		else{
			tmp = "SD=" + hP + attDist;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SK=" + *cLemma + " " + *childP + attDist;
		}
		else{
			tmp = "SK=" + *cLemma + " " + *childP + attDist;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SM=" + *cLemma + attDist;
		}
		else{
			tmp = "SM=" + *cLemma + attDist;
			feat = pool->get(tmp);// this
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SBB=" + hPos;
		}
		else{
			tmp = "SBB=" + hPos;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SDD=" + hP;
		}
		else{
			tmp = "SDD=" + hP;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SKK=" + *cLemma + " " + *childP;
		}
		else{
			tmp = "SKK=" + *cLemma + " " + *childP;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SMM=" + *cLemma;
		}
		else{
			tmp = "SMM=" + *cLemma;
			feat = pool->get(tmp);// this
		}
		flag = add(feat, fv);
		if(flag){
			pool->push_back(feat);
		}
	}
	if (hL > 5) {
		tmp = "SC=" + cPos + attDist;
		feat = pool->get(tmp);
		flag = add(feat, fv);
		if(flag){
			*feat = "SE=" + cP + attDist;
		}
		else{
			tmp = "SE=" + cP + attDist;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SH=" + *hLemma + " " + *headP + attDist;
		}
		else{
			tmp = "SH=" + *hLemma + " " + *headP + attDist;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SJ=" + *hLemma + attDist;
		}
		else{
			tmp = "SJ=" + *hLemma + attDist;
			feat = pool->get(tmp); // this
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SCC=" + cPos;
		}
		else{
			tmp = "SCC=" + cPos;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SEE=" + cP;
		}
		else{
			tmp = "SEE=" + cP;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SHH=" + *hLemma + " " + *headP;
		}
		else{
			tmp = "SHH=" + *hLemma + " " + *headP;
			feat = pool->get(tmp);
		}
		flag = add(feat, fv);
		if(flag){
			*feat = "SJJ=" + *hLemma;
		}
		else{
			tmp = "SJJ=" + *hLemma;
			feat = pool->get(tmp); // this
		}
		flag = add(feat, fv);
		if(flag){
			pool->push_back(feat);
		}
	}
}



